import time
from random import randint
import numpy as np
import pandas as pd
import requests
from lxml import etree
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# 存储爬取的数据
shuju = []


def setup_driver():
    """
    初始化Selenium WebDriver
    :return: WebDriver对象
    """
    options = webdriver.ChromeOptions()
    # 可以在这里添加更多Chrome选项，如无头模式等
    driver = webdriver.Chrome(executable_path='chromedriver.exe', options=options)
    driver.implicitly_wait(120)  # 设置隐式等待时间为120秒
    return driver


def get_product_links(driver, url):
    """
    获取页面上的所有商品链接
    :param driver: WebDriver对象
    :param url: 页面URL
    :return: 商品链接列表
    """
    driver.get(url)
    wait = WebDriverWait(driver, 10)  # 显式等待
    driver.refresh()  # 刷新页面

    # 滚动页面，加载所有商品
    for scroll in range(1, 13):
        js_code = f'window.scrollTo(0, {scroll * 1000})'
        driver.execute_script(js_code)
        time.sleep(randint(1, 3))

    # 等待商品信息加载完成
    wait.until(EC.presence_of_element_located((By.XPATH, '//div[@class="res-info"]')))

    # 获取商品列表
    elements = driver.find_elements(By.XPATH, '//div[@class="res-info"]')
    print(f"找到 {len(elements)} 个元素")

    links = []
    for element in elements:
        link = element.find_element(By.XPATH, './div[@class="title-selling-point"]/a').get_attribute('href')
        if len(link) > 100:
            sku_data = element.find_element(By.XPATH, './div[@class="price-box"]/span').get_attribute('datasku')
            product_id = sku_data.split('|')[0]
            category_id = sku_data.split('|')[5]
            link = f'https://product.suning.com/{category_id}/{product_id}.html?adtype=1'
        links.append(link)

    return links


def extract_product_info(driver, url, headers):
    """
    从商品详情页提取详细信息
    :param driver: WebDriver对象
    :param url: 商品详情页URL
    :param headers: 请求头信息
    :return: 商品详细信息字典
    """
    response = requests.get(url, headers=headers).text
    time.sleep(randint(1, 3))  # 随机等待时间

    # 解析HTML
    html = etree.HTML(response)

    # 抽取商品详细信息
    product_info = {
        '标题': html.xpath('.//div[@class="title-selling-point"]/a/text()')[0].strip(),
        '价格': html.xpath('.//div[@class="price-box"]/span/text()')[0].strip(),
        '评价数': html.xpath('.//div[@class="evaluate-old clearfix"]/div/a/i/text()')[0].strip() if html.xpath(
            './/div[@class="evaluate-old clearfix"]/div/a/i/text()') else np.nan,
        '包装清单': html.xpath('.//tr[@parametercode="999998"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="999998"]/td[2]/text()') else np.nan,
        '品牌': html.xpath('.//tr[@parametercode="000897"]/td[2]/a/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="000897"]/td[2]/a/text()') else np.nan,
        '白酒香型': html.xpath('.//tr[@parametercode="007303"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="007303"]/td[2]/text()') else np.nan,
        '包装': html.xpath('.//tr[@parametercode="006536"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="006536"]/td[2]/text()') else np.nan,
        '酒精度': html.xpath('.//tr[@parametercode="006660"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="006660"]/td[2]/text()') else np.nan,
        '单支容量': html.xpath('.//tr[@parametercode="015839"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="015839"]/td[2]/text()') else np.nan,
        '产地': html.xpath('.//tr[@parametercode="000158"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="000158"]/td[2]/text()') else np.nan,
        '储存方法': html.xpath('.//tr[@parametercode="012753"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="012753"]/td[2]/text()') else np.nan,
        '适用人群': html.xpath('.//tr[@parametercode="007290"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="007290"]/td[2]/text()') else np.nan,
        '适用场景': html.xpath('.//tr[@parametercode="011197"]/td[2]/text()')[0].strip() if html.xpath(
            './/tr[@parametercode="011197"]/td[2]/text()') else np.nan
    }

    return product_info


def save_to_csv(data, filename):
    """
    将数据保存为CSV文件
    :param data: 数据列表
    :param filename: 文件名
    """
    df = pd.DataFrame(data, columns=[
        '标题', '价格', '评价数', '包装清单', '品牌', '白酒香型', '包装', '酒精度',
        '单支容量', '产地', '储存方法', '适用人群', '适用场景'
    ])
    df.to_csv(filename, index=False)


def main():
    """
    主函数，控制整个爬取流程
    """
    headers = {
        'Cookie': 'SN_SESSION_ID=f53ccc85-29b5-40ec-b68c-0e0730c2718b; _snvd=1725690621515bMedXHbpC4S; ...',
        'Host': 'product.suning.com',
        'Referer': 'https://search.suning.com/',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7'
    }
    t = 0

    driver = setup_driver()

    for page in range(0, 28):  # 遍历28页
        url = f'https://search.suning.com/%E9%85%92/&iy=0&isNoResult=0&cp={page}'
        links = get_product_links(driver, url)

        for link in links:
            product_info = extract_product_info(driver, link, headers)
            shuju.append(list(product_info.values()))
            print(list(product_info.values()))
            t += 1
            print(f"爬取了第{t}条")

        save_to_csv(shuju, './苏宁易购酒数据2.csv')

    driver.quit()


if __name__ == '__main__':
    main()